package ffdata.scraping;

import java.io.File;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.By;
import org.openqa.selenium.WebDriver;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.firefox.FirefoxDriver;

import com.google.common.base.Splitter;
import com.google.common.collect.Lists;

import ffdata.utils.FileUtils;
import ffdata.utils.ProgressBar;
import ffmvc.model.JSONHolder;

public class ScrapingMain {

	public static void main(String[] args) {

		// Log in first
		FirefoxDriver driver = logIn(Consts.getPassword(), Consts.getUsername());
		Set<String> hrefs = null;
		File file = null;

		// Get all hrefs to file & print hrefs in href.txt in resources
		if (FileUtils.isFileEmpty(Consts.getHrefsLocation())) {
			hrefs = getPlayerHrefs(driver);
			FileUtils.printHrefs(hrefs);
			file = new File(Consts.getScrapingsLocation()
					+ FileUtils.getNameForScrapingsFile());
			// if hrefs exist, read from file, but first check ih unvisited
			// exist
		} else {
			hrefs = FileUtils.getUnvisitedHrefs();

			if (hrefs == null || hrefs.size() == 0) {
				hrefs = FileUtils.getPlayerHrefsFromFile();
				file = new File(Consts.getScrapingsLocation()
						+ FileUtils.getNameForScrapingsFile());

			} else {

				file = FileUtils.getLatestScrapingFile();
			}

		}

		// main loop for hrefs
		try {
			Double currentPercent = 0.d;
			Double unitPercent = ProgressBar.getUnitPercent(hrefs);
			for (String url : hrefs) {
				currentPercent += unitPercent;

				// add delay to avoid ddos/scraping protection on server side
				try {
					Thread.sleep(3000);
				} catch (InterruptedException e) {

					e.printStackTrace();
				}
				JSONHolder holder = getPlayerStats(url, driver);
				FileUtils.printScrapedData(holder, file);
				// update console
				ProgressBar.updateProgress(currentPercent);

			}
		} catch (Exception e) {
			e.printStackTrace();
		}
		System.out.println("End!");

	}

	private static FirefoxDriver logIn(String pass, String user) {
		// The Firefox driver supports javascript
		FirefoxDriver driver = new FirefoxDriver();

		// Go to the Yahoo login page
		driver.get("https://login.yahoo.com/config/login?.src=spt&.intl=us&.lang=en-US");

		// Login to Yahoo profile
		// Added 5sec interval between
		try {
			driver.findElement(By.id("username")).sendKeys(user);
			Thread.sleep(5000);
			driver.findElement(By.id("passwd")).sendKeys(pass);
			Thread.sleep(5000);
			driver.findElement(By.id(".save")).submit();
			Thread.sleep(5000);
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		return driver;
	}

	private static Set<String> getPlayerHrefs(WebDriver driver) {

		Set<String> hrefList = new HashSet<String>();
		// Go to page with all players
		driver.get(Consts.getPlayersPage());
		// Added 5sec interval between
		try {
			Thread.sleep(10000);
			// Go to advanced search
			driver.findElement(By.className("ys-adv-search")).click();
			Thread.sleep(5000);
			// click Apply
			driver.findElement(By.className("ys-advSearch-btn")).click();
			Thread.sleep(5000);

			// Get all player hrefs
			List<WebElement> anchorList = driver.findElements(By
					.className("ys-player-link"));
			// Retrieve href attribute from anchor
			for (WebElement anchor : anchorList) {
				String href = anchor.getAttribute("href");
				hrefList.add(href);
			}
		} catch (InterruptedException e) {
			e.printStackTrace();

		}
		return hrefList;
	}

	// MAIN PARSING FUNCTION
	private static JSONHolder getPlayerStats(String url, FirefoxDriver driver) {
		// Init Json
		JSONObject playerStats = null;
		JSONObject player = null;
		JSONObject playerHistory = null;

		// Get player page
		driver.get(url);
		String pageSource = (String) driver.getPageSource();
		// Import player page html in jsoup
		Document doc = Jsoup.parse(pageSource);
		// Select all script tags
		Elements scriptElements = doc.getElementsByTag("script");
		// Iterate all script tags looking for player_stats inner text
		for (Element element : scriptElements) {

			if (element.html().contains("root.App.Cache.data.player_stats")) {
				// retrieve variable = value;
				String text = element.html();
				String start = "root.App.Cache.data.player_stats";
				String end = "};";
				Integer indexOfStart = text.indexOf(start);
				Integer indexOfEnd = text.indexOf("};", indexOfStart);
				String substring = text.substring(indexOfStart, indexOfEnd
						+ end.length());

				// now split on =
				Iterable<String> line = Splitter.on('=').trimResults()
						.omitEmptyStrings().split(substring);
				List<String> lineList = Lists.newArrayList(line);
				// get second memeber - after =
				try {
					playerStats = new JSONObject(lineList.get(1));
				} catch (JSONException e) {
					e.printStackTrace();
				}

			}

			if (element.html().contains("root.App.Cache.data.player ")) {
				// retrieve variable = value;
				String text = element.html();
				// IMPORTANT: added space to avoid player_history matching
				String start = "root.App.Cache.data.player ";
				String end = "};";
				Integer indexOfStart = text.indexOf(start);
				Integer indexOfEnd = text.indexOf("};", indexOfStart);
				String substring = text.substring(indexOfStart, indexOfEnd
						+ end.length());

				// now split on =
				Iterable<String> line = Splitter.on('=').trimResults()
						.omitEmptyStrings().split(substring);
				List<String> lineList = Lists.newArrayList(line);
				// get second memeber - after =
				try {
					player = new JSONObject(lineList.get(1));
				} catch (JSONException e) {
					e.printStackTrace();
				}

			}

			if (element.html().contains("root.App.Cache.data.player_history")) {
				// retrieve variable = value;
				String text = element.html();
				String start = "root.App.Cache.data.player_history";
				String end = "};";
				Integer indexOfStart = text.indexOf(start);
				Integer indexOfEnd = text.indexOf("};", indexOfStart);
				String substring = text.substring(indexOfStart, indexOfEnd
						+ end.length());

				// now split on =
				Iterable<String> line = Splitter.on('=').trimResults()
						.omitEmptyStrings().split(substring);
				List<String> lineList = Lists.newArrayList(line);
				// get second memeber - after =
				try {
					playerHistory = new JSONObject(lineList.get(1));
				} catch (JSONException e) {
					e.printStackTrace();
				}

			}

		}
		JSONHolder holder = new JSONHolder();
		holder.setPlayer(player);
		holder.setPlayerHistory(playerHistory);
		holder.setPlayerStats(playerStats);
		holder.setUrl(url);

		return holder;

	}

}
